{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "49036549-3b9f-4ad2-83d6-76dc57c57848",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: transformers in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (4.41.2)\n",
      "Requirement already satisfied: torch in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (2.3.1)\n",
      "Requirement already satisfied: faiss-cpu in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (1.8.0)\n",
      "Requirement already satisfied: filelock in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from transformers) (3.14.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.23.0 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from transformers) (0.23.3)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from transformers) (1.26.4)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from transformers) (23.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from transformers) (6.0.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from transformers) (2024.5.15)\n",
      "Requirement already satisfied: requests in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from transformers) (2.32.2)\n",
      "Requirement already satisfied: tokenizers<0.20,>=0.19 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from transformers) (0.19.1)\n",
      "Requirement already satisfied: safetensors>=0.4.1 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from transformers) (0.4.3)\n",
      "Requirement already satisfied: tqdm>=4.27 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from transformers) (4.66.4)\n",
      "Requirement already satisfied: typing-extensions>=4.8.0 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from torch) (4.11.0)\n",
      "Requirement already satisfied: sympy in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from torch) (1.12.1)\n",
      "Requirement already satisfied: networkx in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from torch) (3.3)\n",
      "Requirement already satisfied: jinja2 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from torch) (3.1.4)\n",
      "Requirement already satisfied: fsspec in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from torch) (2024.6.0)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from torch) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from torch) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from torch) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from torch) (8.9.2.26)\n",
      "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from torch) (12.1.3.1)\n",
      "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from torch) (11.0.2.54)\n",
      "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from torch) (10.3.2.106)\n",
      "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from torch) (11.4.5.107)\n",
      "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from torch) (12.1.0.106)\n",
      "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from torch) (2.20.5)\n",
      "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from torch) (12.1.105)\n",
      "Requirement already satisfied: triton==2.3.1 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from torch) (2.3.1)\n",
      "Requirement already satisfied: nvidia-nvjitlink-cu12 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch) (12.5.40)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from jinja2->torch) (2.1.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from requests->transformers) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from requests->transformers) (2.10)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from requests->transformers) (2.2.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from requests->transformers) (2024.6.2)\n",
      "Requirement already satisfied: mpmath<1.4.0,>=1.1.0 in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (from sympy->torch) (1.3.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers torch faiss-cpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b55f2e07-d64b-41a2-b27a-d73f4ed76ea8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pipeline in /home/thin/.conda/envs/thinp/lib/python3.11/site-packages (0.1.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4e1794c7-53ef-43ec-b93a-1c8445889f08",
   "metadata": {},
   "outputs": [],
   "source": [
    "import faiss\n",
    "import numpy as np\n",
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "dcb3fe77-88d2-495f-af91-3dc1a4bfc807",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "# Load the model and tokenizer for encoding documents and queries\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"meta-llama/Meta-Llama-3-8B-Instruct\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7784c627-69d8-4d95-b844-5a08fa040b2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the padding token\n",
    "tokenizer.pad_token = tokenizer.eos_token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5ae1ce9b-d437-457e-89e0-b3cacd9223dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Move the model to GPU\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2f69e95e-435e-4991-935d-b83a82b08276",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-07-19 10:04:14.920826: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2024-07-19 10:04:14.961893: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-07-19 10:04:15.791565: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4713ca806f524cb5ae9d104cedb40ed0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import transformers\n",
    "\n",
    "# Initialize the text generation pipeline\n",
    "model_id = \"meta-llama/Meta-Llama-3-8B-Instruct\"\n",
    "pipeline = transformers.pipeline(\n",
    "    \"text-generation\",\n",
    "    model=model_id,\n",
    "    model_kwargs={\"torch_dtype\": torch.bfloat16},\n",
    "    tokenizer=tokenizer,\n",
    "    device_map=\"auto\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "b02ead88-3d33-429a-83fc-5b1caef9c645",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the terminators for generation\n",
    "terminators = [\n",
    "    pipeline.tokenizer.eos_token_id,\n",
    "    #pipeline.tokenizer.convert_tokens_to_ids(\"<|eot_id|>\")\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a963a92b-70c1-4077-8fa7-74830fc086ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to generate a response\n",
    "def generate_response(query, SubjectEntityID, relation, Subject, prompt2):\n",
    "    # Retrieve relevant documents\n",
    "\n",
    "    wiki, threshold_cosine = load_wiki(SubjectEntityID, relation, prompt2)\n",
    "    relevant_docs = \"Using this context to answer the question: \"+ wiki\n",
    "    \n",
    "    # Combine the query with the retrieved documents\n",
    "    context = query\n",
    "\n",
    "    # Create the input messages for the generation model\n",
    "    messages = [\n",
    "        {\"role\": \"system\", \"content\": relevant_docs},\n",
    "        {\"role\": \"system\", \"content\": \"You are a chatbot who always responds an answer in english with comma and no explanation. If u don't know the answer, answer None\"},\n",
    "        {\"role\": \"user\", \"content\": context},\n",
    "    ]\n",
    "\n",
    "    # Generate the response\n",
    "    outputs = pipeline(\n",
    "        messages,\n",
    "        max_new_tokens=3000,\n",
    "        eos_token_id=terminators,\n",
    "        do_sample=True,\n",
    "        temperature=0.1,\n",
    "        top_p=0.9,\n",
    "        pad_token_id = pipeline.tokenizer.eos_token_id\n",
    "    )\n",
    "    return outputs[0][\"generated_text\"], wiki, threshold_cosine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3ba22cf9-4b0e-45ec-b651-b75f9b66d6b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def info_valid(text, prompt):\n",
    "\n",
    "    query = '''Is this information \"''' + text + '''\" can give an answer to question: ''' + prompt + \" ?\"\n",
    "    # Create the input messages for the generation model\n",
    "    messages = [\n",
    "        {\"role\": \"system\", \"content\": \"You are a chatbot who always responds an answer with 1 if Yes or 0 if No only\"},\n",
    "        {\"role\": \"user\", \"content\": query}\n",
    "    ]\n",
    "\n",
    "    # Generate the response\n",
    "    outputs = pipeline(\n",
    "        messages,\n",
    "        max_new_tokens=1,\n",
    "        eos_token_id=terminators,\n",
    "        do_sample=True,\n",
    "        temperature=0.1,\n",
    "        top_p=0.9,\n",
    "        pad_token_id = pipeline.tokenizer.eos_token_id\n",
    "    )\n",
    "    return outputs[0][\"generated_text\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e36c73d8-e90a-42bb-9b51-70dc70d642f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('test.jsonl', 'r') as f:\n",
    "    data = [json.loads(line) for line in f]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d6816345-84ec-4475-94d3-c425eebdb4db",
   "metadata": {},
   "outputs": [],
   "source": [
    "from SPARQLWrapper import SPARQLWrapper, JSON\n",
    "\n",
    "def load_url(QID):\n",
    "    sparql = SPARQLWrapper(\"https://query.wikidata.org/sparql\")\n",
    "    sparql.setQuery(f\"\"\"\n",
    "        SELECT ?finalLink \n",
    "        WHERE {{\n",
    "            BIND(wd:{QID} AS ?item)\n",
    "            OPTIONAL {{\n",
    "                ?enwiki schema:about ?item ;\n",
    "                        schema:isPartOf <https://en.wikipedia.org/> .\n",
    "                BIND(?enwiki AS ?finalLink)\n",
    "            }}\n",
    "            OPTIONAL {{\n",
    "                ?otherwiki schema:about ?item ;\n",
    "                        schema:isPartOf ?site .\n",
    "                FILTER (?site != <https://en.wikipedia.org/>)\n",
    "                BIND(?otherwiki AS ?finalLink)\n",
    "            }}\n",
    "            FILTER(BOUND(?finalLink))\n",
    "        }}\n",
    "        ORDER BY ?site\n",
    "        LIMIT 1\n",
    "    \"\"\")\n",
    "    sparql.setReturnFormat(JSON)\n",
    "    results = sparql.query().convert()\n",
    "\n",
    "    if results[\"results\"][\"bindings\"]:\n",
    "        return results[\"results\"][\"bindings\"][0][\"finalLink\"][\"value\"]\n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "952878d9-e32c-4093-b800-52f3a360a979",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/thin/.conda/envs/thinp/lib/python3.11/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from sentence_transformers import SentenceTransformer, util\n",
    "model2 = model = SentenceTransformer('all-MiniLM-L6-v2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5af16f90-2346-4a01-9cad-74531b73b35f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Web Scraping\n",
    "\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "from sentence_transformers import SentenceTransformer, util\n",
    "import time\n",
    "\n",
    "def load_wiki(QID, relation, prompt):\n",
    "    resource = load_url(QID)\n",
    "    if resource is None:\n",
    "        return \"Please use your own llm knowledge.\", [0,0,0]\n",
    "    \n",
    "    res = requests.get(resource)\n",
    "\n",
    "    if res.status_code != 200:\n",
    "        return \"Please use your own llm knowledge.\", [0,0,0]\n",
    "        \n",
    "    soup = BeautifulSoup(res.content, 'html.parser')\n",
    "    heading = soup.find('h1', {'id': 'firstHeading'})\n",
    "    if heading != None:\n",
    "        h = heading.get_text()\n",
    "\n",
    "    documents = []\n",
    "\n",
    "    global model2\n",
    "    prompt_embedding = model2.encode(prompt, convert_to_tensor=True)\n",
    "    \n",
    "    urls = find_link(resource)\n",
    "    url_name = []\n",
    "    for i in range(len(urls)):\n",
    "        name = urls[i].replace(\"https://en.wikipedia.org/wiki/\", \"\").replace(\"_\", \" \")\n",
    "        url_name.append(name)\n",
    "    url_name_embedding = model2.encode(url_name, convert_to_tensor=True)\n",
    "    url_cosine_similarities = util.pytorch_cos_sim(prompt_embedding, url_name_embedding)\n",
    "    url_cosine_similarities_list = url_cosine_similarities.flatten().tolist()\n",
    "    \n",
    "    paragraphs = scrap_para(resource, prompt)\n",
    "    chunk_size = 4500\n",
    "    if paragraphs != None:\n",
    "        for j in range(len(paragraphs)):\n",
    "            if len(paragraphs[j]) > chunk_size:\n",
    "                for k in range(0, len(paragraphs[j]), chunk_size):\n",
    "                    documents.append(paragraphs[j][k:k+chunk_size])\n",
    "            else:\n",
    "                documents.append(paragraphs[j])\n",
    "\n",
    "    infobox = scrap_infobox(resource, prompt)\n",
    "    if infobox != None:\n",
    "        for j in range(len(infobox)):\n",
    "            documents.append(infobox[j])\n",
    "        \n",
    "    table = scrap_table(resource, prompt)\n",
    "    if table != None:\n",
    "        for j in range(len(table)):\n",
    "            documents.append(table[j])\n",
    "\n",
    "\n",
    "    \n",
    "    for i in range(len(urls)):\n",
    "        if url_cosine_similarities_list[i] >= 0.5:\n",
    "            time.sleep(0.01)\n",
    "            resq = requests.get(urls[i])\n",
    "            soup = BeautifulSoup(resq.content, 'html.parser')\n",
    "        \n",
    "            head = soup.find('h1', {'id': 'firstHeading'})\n",
    "            if head != None:\n",
    "                headings = head.get_text()\n",
    "                \n",
    "            para = scrap_para(urls[i], prompt)\n",
    "            if para != None:\n",
    "                for j in range(len(para)):\n",
    "                    if len(para[j]) > chunk_size:\n",
    "                        for k in range(0, len(para[j]), chunk_size):\n",
    "                            #print(para[j][k:k+chunk_size])\n",
    "                            documents.append(para[j][k:k+chunk_size])\n",
    "                    else:\n",
    "                        #print(para[j])\n",
    "                        documents.append(para[j])\n",
    "\n",
    "            infbox = scrap_infobox(urls[i], prompt)\n",
    "            if infbox != None:\n",
    "                for j in range(len(infbox)):\n",
    "                    documents.append(infbox[j])\n",
    "\n",
    "            tabl = scrap_table(urls[i], prompt)\n",
    "            if tabl != None:\n",
    "                for j in range(len(tabl)):\n",
    "                    documents.append(tabl[j])\n",
    "    \n",
    "    document_embeddings = model2.encode(documents, convert_to_tensor=True)\n",
    "    \n",
    "    cosine_similarities = util.pytorch_cos_sim(prompt_embedding, document_embeddings)\n",
    "\n",
    "    cosine_similarities_list = cosine_similarities.flatten().tolist()\n",
    "\n",
    "    sorted_cosine = sorted(cosine_similarities_list, reverse=True)\n",
    "\n",
    "    \n",
    "    maxlen = 20\n",
    "    if len(sorted_cosine) < maxlen:\n",
    "        maxlen = len(sorted_cosine)\n",
    "\n",
    "    threshold_cosine = []\n",
    "    threshold_cosine.append(sorted_cosine[0])\n",
    "    threshold_cosine.append(sorted_cosine[maxlen-1])\n",
    "    threshold_cosine.append(maxlen)\n",
    "    \n",
    "    index_list = []\n",
    "    for i in range(maxlen):\n",
    "        index = cosine_similarities_list.index(sorted_cosine[i])\n",
    "        index_list.append(index)\n",
    "\n",
    "    final_doc = ''\n",
    "    for i in range(maxlen):\n",
    "        final_doc += documents[index_list[i]] + \"\\n\"\n",
    "        \n",
    "    return final_doc, threshold_cosine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f8256875-ffc8-4596-adbe-b8dd3d778758",
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrap_para(url, prompt):\n",
    "\n",
    "    response = requests.get(url)\n",
    "\n",
    "    if response.status_code == 200:\n",
    "        \n",
    "        soup = BeautifulSoup(response.content, \"html.parser\")\n",
    "\n",
    "        paragraphs = soup.find_all('p')\n",
    "\n",
    "        chunk_size = 4500\n",
    "        valid_list = []\n",
    "        for i, paragraph in enumerate(paragraphs):\n",
    "            if len(paragraph.get_text()) > chunk_size:\n",
    "                temp = remove_hyperlinks(paragraph.get_text())\n",
    "                for k in range(0, len(temp), chunk_size):\n",
    "                    if info_valid(temp[k:k+chunk_size], prompt):\n",
    "                        valid_list.append(temp[k:k+chunk_size])\n",
    "            else:\n",
    "                if info_valid(paragraph.get_text(), prompt):\n",
    "                    valid_list.append(remove_hyperlinks(paragraph.get_text()))\n",
    "                \n",
    "    else:\n",
    "        return None\n",
    "\n",
    "\n",
    "    return valid_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8c5f0e96-6d48-4ec4-ae98-45daac1cab0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrap_infobox(url, prompt):\n",
    "    response = requests.get(url)\n",
    "    soup = BeautifulSoup(response.content, \"html.parser\")\n",
    "\n",
    "    contents = ''\n",
    "    # Wiki infobox\n",
    "    datatable = soup.find(\"table\", {\"class\": \"infobox\"})\n",
    "    if datatable is not None:\n",
    "        for tdcol in datatable.select('th[colspan], td[colspan]'):\n",
    "            if tdcol.parent:\n",
    "                tdcol.parent.decompose()\n",
    "\n",
    "        try:\n",
    "            tables = pd.read_html(str(datatable))\n",
    "            if tables:  # Check if any tables were found\n",
    "                df = tables[0]\n",
    "                if not df.empty:\n",
    "                    if isinstance(df.columns, pd.MultiIndex):\n",
    "                        df.columns = df.columns.get_level_values(0)\n",
    "                    df.columns = df.columns.astype(str)  # Convert column names to string\n",
    "                    df = df.loc[:, ~df.columns.str.contains('^Unnamed')]\n",
    "                    datatable_string = ' '.join(df.astype(str).values.flatten())\n",
    "                    contents += ' ' + datatable_string\n",
    "        except ValueError:\n",
    "            # No tables found\n",
    "            pass\n",
    "            \n",
    "    chunk_size = 4500\n",
    "    valid_list = []\n",
    "    contents = remove_hyperlinks(contents)\n",
    "    if len(contents) > chunk_size:\n",
    "        for k in range(0, len(contents), chunk_size):\n",
    "            if info_valid(contents[k:k+chunk_size], prompt):\n",
    "                valid_list.append(contents[k:k+chunk_size])\n",
    "    else:\n",
    "        if info_valid(contents, prompt):\n",
    "            valid_list.append(contents)\n",
    "            \n",
    "    if contents != \"\":\n",
    "        return valid_list\n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a8da3116-e0b6-45da-98ff-4f5bbd75bd6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrap_table(url, prompt):\n",
    "    response = requests.get(url)\n",
    "    soup = BeautifulSoup(response.content, \"html.parser\")\n",
    "    \n",
    "    contents = ''\n",
    "    # Wiki table\n",
    "    wikitable = soup.find(\"table\", {\"class\": \"wikitable\"})\n",
    "    if wikitable is not None:\n",
    "        for tdcol in wikitable.select('th[colspan], td[colspan]'):\n",
    "            if tdcol.parent:\n",
    "                tdcol.parent.decompose()\n",
    "        try:\n",
    "            tables = pd.read_html(str(wikitable))\n",
    "            if tables:  # Check if any tables were found\n",
    "                df = tables[0]\n",
    "                if not df.empty:\n",
    "                    if isinstance(df.columns, pd.MultiIndex):\n",
    "                        df.columns = df.columns.get_level_values(0)\n",
    "                    df.columns = df.columns.astype(str)  # Convert column names to string\n",
    "                    df = df.loc[:, ~df.columns.str.contains('^Unnamed')]\n",
    "                    \n",
    "                    # Concatenate column names with each row's values\n",
    "                    rows = []\n",
    "                    for index, row in df.iterrows():\n",
    "                        row_string = ' '.join([f\"{col}: {row[col]}\" for col in df.columns])\n",
    "                        rows.append(row_string)\n",
    "                    wikitable_string = ' '.join(rows)\n",
    "                    \n",
    "                    contents += ' ' + wikitable_string\n",
    "        except ValueError:\n",
    "            # No tables found\n",
    "            pass\n",
    "            \n",
    "    chunk_size = 4500\n",
    "    valid_list = []\n",
    "    contents = remove_hyperlinks(contents)\n",
    "    if len(contents) > chunk_size:\n",
    "        for k in range(0, len(contents), chunk_size):\n",
    "            if info_valid(contents[k:k+chunk_size], prompt):\n",
    "                valid_list.append(contents[k:k+chunk_size])\n",
    "    else:\n",
    "        if contents != \"\" and info_valid(contents, prompt):\n",
    "            valid_list.append(contents)\n",
    "            \n",
    "    if contents != \"\":\n",
    "        return valid_list\n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "32d72ed3-f29b-48c4-817d-eef5fd643911",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_link(url):\n",
    "    skip_url_list = ['https://', ':']\n",
    "    res = requests.get(url)\n",
    "    soup = BeautifulSoup(res.text, \"html.parser\")\n",
    "    urls = []\n",
    "    for link in soup.find_all(\"a\"):\n",
    "      url = link.get(\"href\", \"\")\n",
    "      if \"/wiki/\" in url and \"https://\" not in url:\n",
    "        if not any(skip_url in url for skip_url in skip_url_list):\n",
    "          url = \"https://en.wikipedia.org\" + url\n",
    "          urls.append(url)\n",
    "\n",
    "    urls = list(dict.fromkeys(list(set(urls))))\n",
    "    return urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "13a23a90-070d-4d7c-8ae7-5a0de46a9ee8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def remove_hyperlinks(text):\n",
    "    # remove_hyperlinks\n",
    "    output_string = re.sub(r'\\[\\d+\\]', '', text)\n",
    "    output_string = re.sub(r'\\[.*?\\]', '', output_string)\n",
    "    output_string = re.sub(r'\\(\\d{4}–\\d{4}\\)', '', output_string)\n",
    "    return output_string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a733e216-7e78-4ee0-8106-aefa3fa17360",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prompt for Knowledge Extraction\n",
    "def load_prompt(SubjectEntity, Relation):\n",
    "    question = ''\n",
    "    if Relation == 'countryLandBordersCountry':\n",
    "        question += \"Which countries share land borders with \" + SubjectEntity + \" with country name only with comma? If None, answer None.\"\n",
    "        \n",
    "    if Relation == 'personHasCityOfDeath':\n",
    "        question += \"What is the city of death of \" + SubjectEntity + \"? answering with one city name only with no explanation. If there is no place of death mentioned, answer None\"\n",
    "\n",
    "    if Relation == 'seriesHasNumberOfEpisodes':\n",
    "        question += \"How many total episodes of series \" + SubjectEntity + \" ? answering with only one number ?\"\n",
    "\n",
    "    if Relation == 'awardWonBy':\n",
    "        question += \"Provide a name only list of all award winners in \" + SubjectEntity + \" with no explanation and name with comma ?\"\n",
    "\n",
    "    if Relation == 'companyTradesAtStockExchange':\n",
    "        question += \"Which stock exchange does \" + SubjectEntity + \" trade on ? answering with no explanation and name with comma? If None, answer None.\"\n",
    "    \n",
    "    return question"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ee19e8ae-0755-4a31-b6ba-0b0630c85e98",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prompt for Web Scraping\n",
    "def load_prompt2(SubjectEntity, Relation):\n",
    "    question = ''\n",
    "    if Relation == 'countryLandBordersCountry':\n",
    "        question += \"Which country share land border with \" + SubjectEntity + \" ?\"\n",
    "        \n",
    "    if Relation == 'personHasCityOfDeath':\n",
    "        question += \"What is the city of death of \" + SubjectEntity + \" ?\"\n",
    "\n",
    "    if Relation == 'seriesHasNumberOfEpisodes':\n",
    "        question += \"How many episodes does series \" + SubjectEntity + \" has ?\" \n",
    "\n",
    "    if Relation == 'awardWonBy':\n",
    "        question += \"Who has won \" + SubjectEntity + \" ?\"\n",
    "\n",
    "    if Relation == 'companyTradesAtStockExchange':\n",
    "        question += \"Which stock exchange does \" + SubjectEntity + \" trade on ?\"\n",
    "    \n",
    "    return question"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "4d575cb0-568c-4533-861d-a21c8d8938e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You seem to be using the pipelines sequentially on GPU. In order to maximize efficiency please use a dataset\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n",
      "/tmp/ipykernel_512907/2959986000.py:14: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(datatable))\n",
      "/tmp/ipykernel_512907/4133377407.py:13: FutureWarning: Passing literal html to 'read_html' is deprecated and will be removed in a future version. To read from a literal string, wrap it in a 'StringIO' object.\n",
      "  tables = pd.read_html(str(wikitable))\n"
     ]
    }
   ],
   "source": [
    "# Finding answers\n",
    "answer = {}\n",
    "web_scrap = []\n",
    "threshold_list = []\n",
    "for i in range(len(data)):\n",
    "    prompt = load_prompt(data[i]['SubjectEntity'], data[i]['Relation'])\n",
    "    prompt2 = load_prompt2(data[i]['SubjectEntity'], data[i]['Relation'])\n",
    "\n",
    "    response, docs, threshold_cosine = generate_response(prompt, data[i]['SubjectEntityID'], data[i]['Relation'], data[i]['SubjectEntity'], prompt2)\n",
    "    answer[prompt] = response[3]['content']\n",
    "    #print(response)\n",
    "    web_scrap.append(docs)\n",
    "    threshold_list.append(threshold_cosine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "10a7f264-1a0f-434e-adf4-81d4ca834fcf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving Results\n",
    "df = pd.DataFrame(list(answer.items()), columns=['Key', 'Value'])\n",
    "\n",
    "df.to_excel('answer.xlsx', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "1b8cf39e-2600-4665-a6f2-b9d3b98c69c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(web_scrap, columns=['Data'])\n",
    "\n",
    "df.to_excel('web_scrap.xlsx', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2ff288e8-0bd9-4197-9185-0c9fd3d41ce4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(threshold_list, columns=['Max Cosine similarity', 'Min Cosine similarity', 'Maxlen'])\n",
    "\n",
    "df.to_excel('cosine.xlsx', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "02a41d5a-e042-4ced-a057-b035739f1070",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
